# Initialize Notebook
from IPython.core.display import HTML,Image
#%run ../library/v1.0.5/init.ipy
HTML('''<script> code_show=true;  function code_toggle() {  if (code_show){  $('div.input').hide();  } else {  $('div.input').show();  }  code_show = !code_show }  $( document ).ready(code_toggle); </script> <form action="javascript:code_toggle()"><input type="submit" value="Toggle Code"></form>''')

import gc, argparse, sys, os, errno
%pylab inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
#sns.set()
#sns.set_style('whitegrid')
import h5py
from PIL import Image
import os
from tqdm import tqdm_notebook as tqdm
import scipy
import sklearn
from scipy.stats import pearsonr
import warnings
warnings.filterwarnings('ignore')
from scipy.io import loadmat
import IPython.display as ipd
import IPython
import librosa.display
import librosa
from pystoi import stoi

Populating the interactive namespace from numpy and matplotlib

/Users/james/anaconda3/lib/python3.7/site-packages/matplotlib/__init__.py:886: MatplotlibDeprecationWarning: 
examples.directory is deprecated; in the future, examples will be found relative to the 'datapath' directory.
  "found relative to the 'datapath' directory.".format(key))

ls

attention_mask.mat
ecog.mat
gradient.mat
gt.wav
merge.wav
myCNN2_fineturn_mergenet_16k_vae_v2_20200420-134911label_test.tsv
pred.wav
result_NY742.Sensory.html
result_NY742.Sensory.ipynb
spectrogram_GT.mat
spectrogram_prediction.mat
waveform_GT.mat

select_word = np.loadtxt([i for i in os.listdir('.') if i[-3:]=='tsv'][0],dtype='str')

Spectrogram

• spec_gt: upper
• spec_pred: bottom

spec_gt = loadmat('spectrogram_GT.mat')['GT_STFT_test_spkr']
spec_pred = loadmat('spectrogram_prediction.mat')['pred_STFT_test']
spec_concat = np.concatenate((numpy.swapaxes(spec_gt,2,1), numpy.swapaxes(spec_pred,2,1)),\
                             axis=1)

row_nums = 18
col_nums = 10
fig,ax=plt.subplots(row_nums,col_nums,figsize=(col_nums*2,row_nums*1.5))
cmap = cm.coolwarm
for i in range(row_nums):
    for j in range(col_nums):
        ax[i,j].imshow(spec_concat[i*col_nums+j] ,cmap=cmap)
        try:
            ax[i,j].set_title(select_word[i*col_nums+j])
        except:
            pass
#fig.suptitle('Spectrogram Demo', fontsize=14)
#fig.subplots_adjust(top=1)
fig.tight_layout()

Waveform

wave_gt = librosa.load('gt.wav',sr=16000)[0]
wave_pred = librosa.load('pred.wav',sr=16000)[0]
wave_merge = librosa.load('merge.wav',sr=16000)[0]

ground truth

display(ipd.Audio(wave_gt,rate=16000))

reconstructed audio

display(ipd.Audio(wave_pred,rate=16000))

merged audio

display(ipd.Audio(wave_merge,rate=16000))

interval = 16384
row_nums = 18
col_nums = 10
fig,ax=plt.subplots(row_nums*2,col_nums,figsize=(col_nums*2,row_nums*1.5))
for i in range(row_nums):
    for j in range(col_nums):
        try:
            ax[i*2,j].set_title(select_word[i*col_nums+j])
        except:
            pass
        ax[i*2,j].plot(wave_gt[(i*col_nums+j)*interval:(i*col_nums+j+1)*interval])
        ax[i*2+1,j].plot(wave_pred[(i*col_nums+j)*interval:(i*col_nums+j+1)*interval])
        #librosa.display.waveplot(wave_gt[(i*col_nums+j)*interval:(i*col_nums+j+1)*interval], sr=16000,ax=ax[i*2,j])
        #librosa.display.waveplot(wave_pred[(i*col_nums+j)*interval:(i*col_nums+j+1)*interval], sr=16000,ax=ax[i*2+1,j])
        ax[i*2,j].axis('off')
        ax[i*2+1,j].axis('off')
fig.tight_layout()

Visualization

Attention

attention = loadmat('attention_mask.mat')['ams_test'][:,:,:,:,0]
average_mask = attention.mean(axis=0)
average_mask.shape

(36, 15, 15)

row_nums = 3
col_nums = 12
fig,ax=plt.subplots(row_nums,col_nums,figsize=(col_nums*3,row_nums*3))
for i in range(row_nums):
    for j in range(col_nums):
        im = ax[i,j].imshow(average_mask[i*col_nums+j],cmap=cm.hot,vmin=mean(\
                    average_mask[average_mask!=0]),\
                   vmax=np.max(average_mask))
        ax[i,j].axis('off')
        
        
fig.subplots_adjust(right=0.84)
cbar_ax = fig.add_axes([0.85, 0.15, 0.02, 0.7])
fig.colorbar(im, cbar_ax)

<matplotlib.colorbar.Colorbar at 0x13ffbf210>

Ecog

ecog = loadmat('ecog.mat')['GT_STFT_test_ecog'][0,:,:,:].reshape(180,176, 15,15)[:,16:-16,:,:]
ecog_ = np.zeros([36,15,15])
for i in range(36):
    ecog_[i] = np.mean(np.max(ecog[:,i*4:(i+1)*4,:,:],1),axis=0)

row_nums = 3
col_nums = 12
fig,ax=plt.subplots(row_nums,col_nums,figsize=(col_nums*3,row_nums*3))
for i in range(row_nums):
    for j in range(col_nums):
        im = ax[i,j].imshow(ecog_[i*col_nums+j],cmap=cm.hot,vmin=mean(\
                    ecog_[ecog_!=0]),\
                   vmax=np.max(ecog_))
        ax[i,j].axis('off')
        
fig.subplots_adjust(right=0.84)
cbar_ax = fig.add_axes([0.85, 0.15, 0.02, 0.7])
fig.colorbar(im, cbar_ax)

<matplotlib.colorbar.Colorbar at 0x1415041d0>

Gradient

gradient = loadmat('gradient.mat')['grad_loss2inp_test'].reshape(-1,\
                        176, 225).reshape(-1,176, 15,15)[:,16:-16,:,:]
gradient[gradient<0] = 0
gradient = np.abs(gradient)
gradient_ = np.zeros([36,15,15])
for i in range(36):
    gradient_[i] = np.mean(np.max(gradient[:,i*4:(i+1)*4,:,:],1),axis=0)

row_nums = 3
col_nums = 12
fig,ax=plt.subplots(row_nums,col_nums,figsize=(col_nums*3,row_nums*3))
for i in range(row_nums):
    for j in range(col_nums):
        im = ax[i,j].imshow(gradient_[i*col_nums+j],cmap=cm.hot,vmin=mean(\
                    gradient_[gradient_!=0]),\
                   vmax=np.max(gradient_))
        ax[i,j].axis('off')
        
fig.subplots_adjust(right=0.84)
cbar_ax = fig.add_axes([0.85, 0.15, 0.02, 0.7])
fig.colorbar(im, cbar_ax)

<matplotlib.colorbar.Colorbar at 0x135b19390>

Metrics

PCC&MSE

def MSE_pcc(A,B,ax=None):
    mse =np.mean(((A - B)**2/B.var()))
    pcc = pearsonr(A.ravel(),B.ravel())[0]
    return mse,pcc
def analyze(predict,GT_STFT_test_spkr):
    samples = predict.shape[0]
    pcc = np.zeros([samples])
    mse = np.zeros([samples])
    for i in range(samples):
        mse[i], pcc[i] = MSE_pcc(predict[i],GT_STFT_test_spkr[i])
    fig,ax=plt.subplots(1,2,figsize=(16,4))
    ax[0].hist(mse,bins=25,color='b')
    ax[0].set_title('MSE: %g(%g)' %(np.round(mse.mean(),3),np.round(mse.std(),3)))
    ax[1].hist(pcc,bins=50,color='g')
    ax[1].set_title('PCC: %g(%g)' %(np.round(pcc.mean(),3),np.round(pcc.std(),3)))
    return mse,pcc

spec_gt = loadmat('spectrogram_GT.mat')['GT_STFT_test_spkr']
spec_pred = loadmat('spectrogram_prediction.mat')['pred_STFT_test']
mse,pcc = analyze(spec_pred,spec_gt)

STOI

stois = np.zeros([180])
for i in range(180):
    stois[i]=stoi(wave_gt[i*interval:(i+1)*interval], wave_pred[i*interval:(i+1)*interval], \
                      16000, extended=False)

fig,ax=plt.subplots(figsize=(8,4))
ax.hist(stois,bins=25,color='b')
ax.set_title('STOI: %g(%g)' %(np.round(stois.mean(),3),np.round(stois.std(),3)))

Text(0.5, 1.0, 'STOI: 0.438(0.157)')